import time
from concurrent.futures import ThreadPoolExecutor
import requests
import datetime
from requests.adapters import HTTPAdapter


def __get_time():
    current_time = datetime.datetime.now().strftime('%Y-%m-%d %X')
    return current_time


def time_me(fn):
    def _wrapper(*args, **kwargs):
        start = datetime.datetime.now()	 # 获取当前时间
        ret = fn(*args, **kwargs)
        end = datetime.datetime.now()
        print(f"[{__get_time()}] --->function:{fn.__name__} {args} {kwargs} cost {(end-start).seconds} seconds")
        return ret
    return _wrapper

@time_me
def get_gitee_request(url, flag):
    requests.adapters.DEFAULT_RETRIES = 10
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36',
        'Accept': '*/*',
        'Accept-Encoding': 'gzip, deflate, br',
        'Accept-Language': 'zh-CN,zh;q=0.9',
        'Connection': 'keep-alive',
        'Host': 'gitee.com',
        'X-Requested-With': 'XMLHttpRequest',
    }
    try:
        resp = requests.get(url, headers=headers)
    except requests.exceptions.RequestException as e:
        print(f"spider {url} {headers} Exception")
        return None
    return flag, resp

@time_me
def get_github_api_request(url, flag):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36',
        'Accept': 'application/json',
        'Authorization': 'token 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',
        'Accept-Encoding': 'gzip, deflate, br',
        'Accept-Language': 'zh-CN,zh;q=0.8,en;q=0.6,zh-TW;q=0.4',
        'X-Requested-With': 'XMLHttpRequest',
        'Pragma': 'no-cache',
    }
    try:
        requests.adapters.DEFAULT_RETRIES = 30
        s = requests.session()
        s.keep_alive = False
        resp = requests.get(url, headers=headers, timeout=100)
    except requests.exceptions.RequestException as e:
        print(f"spider {url} {headers} Exception")
        return None, None
    # time.sleep(40)
    return flag, resp
    # time.sleep(45)


def __action(url: str, flag: dict, index: int):
    if index == 0:
        response = get_gitee_request(url, flag)
    elif index == 1:
        # time.sleep(40)
        response = get_github_api_request(url, flag)
        time.sleep(40)
    # time.sleep(1)
    return response


def spider(max_workers: int, tasks: list, index: int):
    result = list()
    futures = list()
    with ThreadPoolExecutor(max_workers) as pool:
        for task in tasks:
            url = task[0]
            flag = task[1]
            futures.append(pool.submit(__action, url, flag, index))
    for future in futures:
        result.append(future.result())
    return result

def __main():
    url = "https://gitee.com/explore/all?order=starred"
    tasks = list()
    for page in range(1):
        # param = dict()
        # param["access_token"] = "1b0038e8549006c6f55fc654cf43f6de"
        # param["page"] = page
        # param["per_page"] = 15
        flag = ['全部项目']
        # param["type"] = "all"
        task = (url, flag)
        tasks.append(task)
    ret = spider(20, tasks)
    print(ret)

if __name__ == "__main__":
    __main()